#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    :
# @Author  :
# @File    : 内容解析.py
# @Software: PyCharm

import scrapy
# 进入到项目目录下，使用scrapy genspider stock quote.stockstar.com  语句进行创建
# 次文件定义了爬虫的起始地址，并创建了parse的自定义函数，在parse函数中书写
# 此文件下进行爬虫逻辑的编写
from scrapyBook.items import ScrapybookItem

"""
    #1.先从https://book.qidian.com/info/1026225232/#Catalog 中获取43章所有的链接最后章节地址
    #2.拼接需要解析的地址，循环解析写入对象
        #https://read.qidian.com/chapter/NRLZdl9YG2KXfJNNZ-YUzw2/VT8ViOMzY3hMs5iq0oQwLQ2/
    #3.根据列表循环获取文本内容，保存在list中，标题、正文、数字（有序的数字）
    #4.在pipelines中将list排序，并解析获取，将数据写入到txt中。
"""


class StockSpider(scrapy.Spider):
    #名称
    name = 'stock'
    #域名
    allowed_domains = ['read.qidian.com']
    #解析网站
    start_urls = ['https://book.qidian.com/info/1026225232/#Catalog']


    counts = 0
    fictionItem = ScrapybookItem()
    #解析列表获取链接
    def parse(self, response):
        count = 0
        urlContent = response.xpath("//ul[@class='cf']/li")
        for li in urlContent:
            hrefText = li.xpath("./h2/a/@href").get().split("/")[5]
            next_url = "https://read.qidian.com/chapter/NRLZdl9YG2KXfJNNZ-YUzw2/" + hrefText +"/"
            sectionCount = li.xpath("./@data-rid").get()
            yield scrapy.Request(next_url, callback=self.parseContent, dont_filter=False,meta={'sectionCount':sectionCount})
            count += 1
            if count == 3:
                break



    #解析正文
    def parseContent(self,response):

        #获取章节参数
        sectionCount = response.meta['sectionCount']
        # 保存章节序号
        self.fictionItem['sectionCount'] = int(sectionCount)

        #解析章节
        sectionText = response.xpath("//span[@class='content-wrap']/text()")
        #保存章节名称
        section = sectionText.get()
        self.fictionItem['section'] = section

        #解析正文
        mainBodyTextlist = response.xpath("//div[@class='read-content j_readContent']/p")
        mainBody = ""
        #遍历正文
        for a in mainBodyTextlist:
            mainBody = mainBody + a.xpath("./text()").get() + '\n'
        #保存内容
        self.fictionItem['mainBody'] = mainBody

        #触发保存
        yield self.fictionItem

